package cn.net.withub.dataCollector.web.collector;

import cn.edu.hfut.dmic.webcollector.model.CrawlDatum;
import cn.edu.hfut.dmic.webcollector.model.CrawlDatums;
import cn.edu.hfut.dmic.webcollector.model.Page;
import cn.edu.hfut.dmic.webcollector.plugin.berkeley.BreadthCrawler;
import cn.net.withub.dataCollector.common.editor.DateUtil;
import cn.net.withub.dataCollector.common.model.*;
import cn.net.withub.dataCollector.common.utils.CreateNewKey;
import cn.net.withub.dataCollector.common.utils.HTMLSpirit;
import cn.net.withub.dataCollector.web.service.CollectorService;
import org.apache.log4j.Logger;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.web.context.WebApplicationContext;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;

/**
 * Created by yuanjie on 2018/4/25.
 */
public class CollectorUtilsM17 extends BreadthCrawler {

    //@Resource
    private CollectorService collectorService;

    //@Resource
    private WebApplicationContext webAppliction;

    private TCollectorConfig tCollectorConfig;

    private Logger log = Logger.getLogger(this.getClass());
    //private SimpleDateFormat new SimpleDateFormat("yyyy-MM-dd") = new SimpleDateFormat("yyyy-MM-dd");

    public void CollectorUtils(CollectorService collectorService){
        collectorService = collectorService;
    }

    public CollectorUtilsM17(String crawlPath, boolean autoParse, WebApplicationContext wac, TCollectorConfig config) {
        super(crawlPath, autoParse);

        webAppliction = wac;
        if(webAppliction !=null){
            collectorService = (CollectorService)webAppliction.getBean("collectorService");
        }
        if(config != null){
            tCollectorConfig = config;
        }
    }

    public void visit(Page page, CrawlDatums crawlDatums) {
        String url = page.getUrl();
        String html = page.getHtml();
        String contentType = page.getResponse().getContentType();
        //Document doc = page.getDoc();
        log.info("--------------------------URL:\n" + url+"     "+contentType);
        try{
            //if(page.matchUrl("http://149.4.0.202/information/infolist.asp\\?classid=.*") || page.matchUrl("http://149.4.0.202/information/infolist.asp\\?p=.*&classid=.*")){
            if(page.matchUrl("http://.*/information/infolist.asp\\?classid=.*") || page.matchUrl("http://.*/information/infolist.asp\\?p=.*&classid=.*")){
                //log.info("--------------------------html:\n"+page.getHtml());
                //获取内容
                Elements trElements = page.select("tr[bgcolor]:has(a[href][target])");//[target=_blank]
                for (Element tr : trElements) {
                    String fbsj = "";
                    String title = "";
                    String src = "";
                    try{
                        //获取发布时间
                        Elements tdElements = tr.select("td[width=15%]");
                        if(tdElements != null && tdElements.size()>0){
                            fbsj = tdElements.get(0).text();
                        }
                        fbsj = fbsj.replace(".","-").replace("[","").replace("]","");
                        //log.info("fbsj...."+fbsj);

                        //获取发布标题
                        Elements titleElements = tr.select("td[width=85%] a[href][target]");//[target=_blank]
                        if(titleElements != null && titleElements.size()>0){
                            title = titleElements.get(0).text();
                        }
                        //log.info("title...."+title);

                        Elements elements = tr.select("a[href][target]");
                        //log.info("-----------href url size:"+elements.size()+"........."+tr.select("td a[href][target]").size());
                        for (Element element : elements) {
                            src = element.attr("abs:href");
                            log.info("href url:\n"+src);
                            if(tCollectorConfig.getFbsj()==null || (tCollectorConfig.getFbsj()!=null && DateUtil.compareDate(fbsj,new SimpleDateFormat("yyyy-MM-dd").format(tCollectorConfig.getFbsj()))>0)){
                                log.info("fbsj: "+fbsj);
                                crawlDatums.add(
                                        new CrawlDatum(src)
                                                .meta("title",title)
                                                .meta("fbsj",fbsj)
                                                .meta("firstUrl",url)
                                );
                            }
                            //判断已抓取过的最大发布日期，根据标题和发布时间进行对比，未抓取过的进行抓取
                            else if(tCollectorConfig.getFbsj()!=null && DateUtil.compareDate(fbsj,new SimpleDateFormat("yyyy-MM-dd").format(tCollectorConfig.getFbsj()))==0){
                                log.info("fbsj: "+fbsj+"相同");
                                TCollectorData tCollectorData = (TCollectorData)collectorService.load(TCollectorData.class," fbsj='"+fbsj+"' and title='"+title+"'");
                                if(tCollectorData == null){
                                    log.info("fbsj: "+fbsj+"抓取");
                                    crawlDatums.add(
                                            new CrawlDatum(src)
                                                    .meta("title",title)
                                                    .meta("fbsj",fbsj)
                                                    .meta("firstUrl",url)
                                    );
                                }
                            }

                        }
                    }catch (Exception e){
                        saveLog(title,fbsj,url,src,0);
                        e.printStackTrace();
                        continue;
                    }
                }


                //获取下一页
                String src = "";
                try{
                    Elements pageElements = page.select("a[class=white_bg]");
                    for(Element element : pageElements){
                        src = element.attr("abs:href");
                        String text = element.text().trim();
                        if(text.equals("下一页")){
                            log.info("抓取下一页......"+src);
                            crawlDatums.add(
                                    new CrawlDatum(src)
                            );
                            /*CollectorM00Task task = (CollectorM00Task)webAppliction.getBean("collectorM00Task");
                            log.info(task);
                            task.collectorYzxw(tCollectorConfig);
                            */
                        }
                    }
                }catch (Exception e){
                    saveLog("","",url,src,0);
                    e.printStackTrace();
                }
            }
            //else if(page.matchUrl("http://149.4.0.202/information/InformationDisplay.asp\\?NewsID=.*")){
            else if(page.matchUrl("http://.*/information/InformationDisplay.asp\\?NewsID=.*")){
                log.info("contentType:     \n"+contentType+"     meta:"+page.meta("title"));
                Map<String,Object> map = new HashMap<String, Object>();
                String content = "";
                String serverFilePath = "";
                String wjgs = "";

                String title = page.meta("title");
                String fbsj = page.meta("fbsj");
                String firstUrl = page.meta("firstUrl");
                int zt = 0;

                if(contentType.contains("msword") || contentType.contains("pdf")){
                    try{
                        //获取word文件
                        if(contentType.contains("msword")){
                            map = HttpClientUtils.getWord(url);
                            wjgs = "doc";
                        }
                        //获取pdf文件
                        else if(contentType.contains("pdf")){
                            map = HttpClientUtils.getPdf(url);
                            wjgs = "pdf";
                        }

                        if(map != null){
                            content = map.get("content")==null?"":map.get("content").toString();
                            serverFilePath = map.get("serverFilePath")==null?"":map.get("serverFilePath").toString();
                        }

                        Date date = new Date();
                        String id = new CreateNewKey().createId();
                        TCollectorData tCollectorData = new TCollectorData();
                        tCollectorData.setId(id);
                        tCollectorData.setConfigId(tCollectorConfig.getId());
                        tCollectorData.setTitle(title);
                        tCollectorData.setFbsj(new SimpleDateFormat("yyyy-MM-dd").parse(fbsj));
                        tCollectorData.setWjgs(wjgs);
                        tCollectorData.setUrl(url);
                        tCollectorData.setFydm(tCollectorConfig.getFydm());
                        tCollectorData.setFymc(tCollectorConfig.getFymc());
                        tCollectorData.setWjlj(serverFilePath);
                        tCollectorData.setSystime(date);
                        collectorService.save(tCollectorData);

                        TCollectorContent tCollectorContent = new TCollectorContent();
                        tCollectorContent.setId(new CreateNewKey().createId());
                        tCollectorContent.setContent(content);
                        tCollectorContent.setSystime(date);
                        tCollectorContent.setDataId(id);
                        collectorService.save(tCollectorContent);

                        //保存最大发布时间
                        TCollectorConfig config = (TCollectorConfig)collectorService.load(TCollectorConfig.class," id='"+tCollectorConfig.getId()+"'");
                        if(config.getFbsj()==null || DateUtil.compareDate(fbsj,new SimpleDateFormat("yyyy-MM-dd").format(config.getFbsj()))>0){
                            log.info("config........."+fbsj);
                            config.setFbsj(new SimpleDateFormat("yyyy-MM-dd").parse(fbsj));
                            collectorService.save(config);
                        }

                        zt = 1;
                    }catch(Exception e){
                        zt = 0;
                        e.printStackTrace();
                    }finally {
                        saveLog(title,fbsj,firstUrl,url,zt);
                    }
                }
                //html文件
                else if(contentType.contains("html")){
                    try{
                        //获取发布相关信息
                        //String ttt = page.select("td[class=bodytitle]>strong").first().text();
                        String ttt1 = page.select("td[class=bodytd2]").text();
                        //int index_fbsj = ttt1.indexOf("发布时间:");
                        int index_bjsh = ttt1.indexOf("作者:");
                        if(index_bjsh <= -1){
                            index_bjsh = ttt1.indexOf("发布者:");
                            if(index_bjsh <= -1){
                                index_bjsh = ttt1.indexOf("发布者：");
                            }
                        }
                        int index_djcs = ttt1.indexOf("点击次数：");

                        String bjsh = ttt1.substring(index_bjsh+3,index_djcs);
                        String djcs = ttt1.substring(index_djcs+5);

                        //获取iframe内容
                        Elements iframe=page.select("iframe[id=ifrName]");
                        String src = iframe.attr("abs:src");
                        if(src.contains("UpDocument")){
                            src = src.replace("/Information/","").replace("/information/","").replace("\\","/");
                        }

                        //重新添加爬取任务
                        //crawlDatums.add(src);
                        crawlDatums.add(new CrawlDatum(src)
                                .meta("title",title)
                                .meta("fbsj",fbsj)
                                .meta("bjsh",bjsh)
                                .meta("djcs",djcs)
                                .meta("firstUrl",firstUrl)
                                .meta("tqUrl",url)
                        );
                    }catch (Exception e){
                        saveLog(title,fbsj,firstUrl,url,0);
                        e.printStackTrace();
                    }
                }
            }
            //else if(page.matchUrl("http://149.4.0.202/information/displaycont.asp\\?newsid=.*")){
            else if(page.matchUrl("http://.*/information/displaycont.asp\\?newsid=.*")){
                //log.info("--------------------------displaycont:\n"+page.getHtml());
                //保存爬取信息
                String title = page.meta("title")==null?"":page.meta("title");
                String fbsj = page.meta("fbsj")==null?"":page.meta("fbsj");
                String bjsh = page.meta("bjsh")==null?"":page.meta("bjsh");
                String djcs = page.meta("djcs")==null?"":page.meta("djcs");
                String firstUrl = page.meta("firstUrl")==null?"":page.meta("firstUrl");
                String tqUrl = page.meta("tqUrl")==null?"":page.meta("tqUrl");
                int zt = 0;

                try{
                    Date date = new Date();
                    String id = new CreateNewKey().createId();
                    TCollectorData tCollectorData = new TCollectorData();
                    tCollectorData.setId(id);
                    tCollectorData.setConfigId(tCollectorConfig.getId());
                    tCollectorData.setTitle(title);
                    tCollectorData.setFbsj(new SimpleDateFormat("yyyy-MM-dd").parse(fbsj));
                    tCollectorData.setXxly("");
                    tCollectorData.setXxfb("");
                    tCollectorData.setBjsh(bjsh);
                    tCollectorData.setDjcs(djcs);
                    tCollectorData.setWjgs("html");
                    tCollectorData.setUrl(tqUrl);
                    tCollectorData.setFydm(tCollectorConfig.getFydm());
                    tCollectorData.setFymc(tCollectorConfig.getFymc());
                    tCollectorData.setSystime(date);
                    collectorService.save(tCollectorData);

                    TCollectorContent tCollectorContent = new TCollectorContent();
                    tCollectorContent.setId(new CreateNewKey().createId());
                    tCollectorContent.setContent(html);
                    tCollectorContent.setText(HTMLSpirit.getTextFromTHML(html));
                    tCollectorContent.setSystime(date);
                    tCollectorContent.setDataId(id);
                    collectorService.save(tCollectorContent);

                    //遍历所有html中所有图片
                    Elements imgs = page.select("img[src]");
                    for (Element img : imgs) {
                        String src = img.attr("abs:src").replace("../","");
                        log.info("img url:\n"+src);
                        crawlDatums.add(new CrawlDatum(src)
                                        .meta("dataId",id)
                        );
                    }

                    //保存最大发布时间
                    TCollectorConfig config = (TCollectorConfig)collectorService.load(TCollectorConfig.class," id='"+tCollectorConfig.getId()+"'");
                    if(config.getFbsj()==null || DateUtil.compareDate(fbsj,new SimpleDateFormat("yyyy-MM-dd").format(config.getFbsj()))>0){
                        config.setFbsj(new SimpleDateFormat("yyyy-MM-dd").parse(fbsj));
                        collectorService.save(config);
                    }

                    zt = 1;
                }catch(Exception e){
                    zt = 0;
                    e.printStackTrace();
                }finally {
                    saveLog(title,fbsj,firstUrl,tqUrl,zt);
                }
            }
            else if(page.matchUrl("http://.*/UpDocument/.*.htm")){
                //http://149.98.0.4/UpDocument/20180607/201806071428347285.htm
                //log.info("--------------------------displaycont:\n"+page.getHtml());
                //保存爬取信息
                String title = page.meta("title")==null?"":page.meta("title");
                String fbsj = page.meta("fbsj")==null?"":page.meta("fbsj");
                String bjsh = page.meta("bjsh")==null?"":page.meta("bjsh");
                String djcs = page.meta("djcs")==null?"":page.meta("djcs");
                String firstUrl = page.meta("firstUrl")==null?"":page.meta("firstUrl");
                String tqUrl = page.meta("tqUrl")==null?"":page.meta("tqUrl");
                int zt = 0;

                try{
                    Date date = new Date();
                    String id = new CreateNewKey().createId();
                    TCollectorData tCollectorData = new TCollectorData();
                    tCollectorData.setId(id);
                    tCollectorData.setConfigId(tCollectorConfig.getId());
                    tCollectorData.setTitle(title);
                    tCollectorData.setFbsj(new SimpleDateFormat("yyyy-MM-dd").parse(fbsj));
                    tCollectorData.setXxly("");
                    tCollectorData.setXxfb("");
                    tCollectorData.setBjsh(bjsh);
                    tCollectorData.setDjcs(djcs);
                    tCollectorData.setWjgs("html");
                    tCollectorData.setUrl(tqUrl);
                    tCollectorData.setFydm(tCollectorConfig.getFydm());
                    tCollectorData.setFymc(tCollectorConfig.getFymc());
                    tCollectorData.setSystime(date);
                    collectorService.save(tCollectorData);

                    TCollectorContent tCollectorContent = new TCollectorContent();
                    tCollectorContent.setId(new CreateNewKey().createId());
                    tCollectorContent.setContent(html);
                    tCollectorContent.setText(HTMLSpirit.getTextFromTHML(html));
                    tCollectorContent.setSystime(date);
                    tCollectorContent.setDataId(id);
                    collectorService.save(tCollectorContent);

                    //遍历所有html中所有图片
                    Elements imgs = page.select("img[src]");
                    for (Element img : imgs) {
                        String src = img.attr("abs:src").replace("../","");
                        log.info("img url:\n"+src);
                        crawlDatums.add(new CrawlDatum(src)
                                .meta("dataId",id)
                        );
                    }

                    //保存最大发布时间
                    TCollectorConfig config = (TCollectorConfig)collectorService.load(TCollectorConfig.class," id='"+tCollectorConfig.getId()+"'");
                    if(config.getFbsj()==null || DateUtil.compareDate(fbsj,new SimpleDateFormat("yyyy-MM-dd").format(config.getFbsj()))>0){
                        config.setFbsj(new SimpleDateFormat("yyyy-MM-dd").parse(fbsj));
                        collectorService.save(config);
                    }

                    zt = 1;
                }catch(Exception e){
                    zt = 0;
                    e.printStackTrace();
                }finally {
                    saveLog(title,fbsj,firstUrl,tqUrl,zt);
                }
            }
            else if(contentType !=null && contentType.startsWith("image")){
                String dataId = page.meta("dataId")==null?"":page.meta("dataId");
                String imageName = url.substring(url.lastIndexOf("/")+1,url.length());
                //log.info("imageName:\n"+imageName);
                String serverFilePath = "";

                try {
                    serverFilePath = HttpClientUtils.getImage(url);

                    TCollectorElement tCollectorElement = new TCollectorElement();
                    tCollectorElement.setId(new CreateNewKey().createId());
                    tCollectorElement.setDataId(dataId);
                    tCollectorElement.setTitle(imageName);
                    tCollectorElement.setUrl(url);
                    tCollectorElement.setWjlj(serverFilePath);
                    tCollectorElement.setSystime(new Date());
                    collectorService.save(tCollectorElement);
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }catch (Exception e){
            e.printStackTrace();
        }
    }

    public void saveLog(String title,String fbsj,String firstUrl,String tqUrl,int zt){
        try {
            log.info("savelog........."+fbsj);
            TCollectorLog tCollectorLog = new TCollectorLog();
            tCollectorLog.setId(new CreateNewKey().createId());
            tCollectorLog.setConfigId(tCollectorConfig.getId());
            tCollectorLog.setIp(tCollectorConfig.getIp());
            tCollectorLog.setModuleName(tCollectorConfig.getModuleName());
            tCollectorLog.setFydm(tCollectorConfig.getFydm());
            tCollectorLog.setFymc(tCollectorConfig.getFymc());
            tCollectorLog.setTitle(title);
            try {
                tCollectorLog.setFbsj(new SimpleDateFormat("yyyy-MM-dd").parse(fbsj));
            }catch (Exception e) {
                e.printStackTrace();
            }
            tCollectorLog.setZt(zt);
            tCollectorLog.setFirstUrl(firstUrl);
            tCollectorLog.setTqUrl(tqUrl);
            tCollectorLog.setSysTime(new Date());

            collectorService.save(tCollectorLog);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    /**
     * 向文本文件中写入内容或追加新内容,如果append为true则直接追加新内容,<br>
     * 如果append为false则覆盖原来的内容<br>
     *
     * @param path
     * @param content
     * @param append
     * @throws IOException
     */
    public static void writeFile(String path, byte[] content, boolean append)
            throws IOException {
        // 通过这个对象来判断是否向文本文件中追加内容
        File writefile = new File(path);
        // 如果文本文件不存在则创建它
        if (writefile.exists() == false) {
            if(mkDirectory(path.substring(0, path.lastIndexOf("/")))){
                writefile.createNewFile();
                writefile = new File(path); // 重新实例化
            }
        }
        FileOutputStream fw = new FileOutputStream(writefile);
        fw.write(content);
        fw.flush();
        fw.close();
    }

    /**
     * 根据路径创建一系列的目录
     * @param path 完整路径
     * @return 是否成功
     */
    public static boolean mkDirectory(String path){
        File file = null;
        try{
            file = new File(path);
            if (!file.exists()){
                return file.mkdirs();
            }
        }catch (RuntimeException e){
            e.printStackTrace();
        }finally{
            file = null;
        }
        return false;
    }

    public static void main(String[] args) throws Exception {
        CollectorUtilsM17 crawler = new CollectorUtilsM17("crawl", true,null,null);
        crawler.setThreads(50);
        crawler.setTopN(100);
        //crawler.setResumable(true);
        crawler.start(300);

        /*Executor executor=new Executor() {
            public void execute(CrawlDatum datum, CrawlDatums next) throws Exception {
                HtmlUnitDriver driver = new HtmlUnitDriver();
                driver.setJavascriptEnabled(true);
                driver.get(datum.getUrl());
                //WebElement element=driver.findElementByCssSelector("ul#ssxw");
                WebElement element=driver.findElementByCssSelector("span#outlink1");
                log.info("反链数:"+element.getText());
            }
        };

        //创建一个基于伯克利DB的DBManager
        DBManager manager=new BerkeleyDBManager("crawl");
        //创建一个Crawler需要有DBManager和Executor
        Crawler crawler= new Crawler(manager,executor);
        //crawler.addSeed("http://149.0.160.10");
        crawler.addSeed("http://seo.chinaz.com/?host=www.tuicool.com");
        crawler.start(1);*/
    }
}
